'''
Created on 2011-3-2

@author: summit
'''

import urllib, urllib2

from BeautifulSoup import BeautifulSoup, ICantBelieveItsBeautifulSoup, MinimalSoup

from pdfminer.pdfparser import PDFParser, PDFDocument, PDFNoOutlines
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfdevice import PDFDevice
from pdfminer.converter import PDFPageAggregator
from pdfminer.layout import LAParams, LTTextBox, LTTextLine, LTFigure, LTImage

from nltk.tokenize import sent_tokenize


import sys
import os
from binascii import b2a_hex



class JGPdf:
    '''
    We use the pdfminer to anlysis an pdf ebook
    '''
    
    def __init__(self, file, password=''):
        # Open a pdf file
        self.fp = open(file, 'rb')
        # Create a PDF parser object associated with the file object.
        self.parser = PDFParser(self.fp)
        # Create a PDF document object that stores the document structure.
        self.doc = PDFDocument()
        # Connect the parser and document objects.
        self.parser.set_document(self.doc)
        self.doc.set_parser(self.parser)
        # Supply the password for initialization.
        # (If no password is set, give an empty string.)
        self.doc.initialize(password)   # must have exec this command
        # Check if the document allows text extraction. If not, abort.
        if not self.doc.is_extractable:
            raise PDFTextExtractionNotAllowed('Text extraction is not allowed: %r' % self.fp)
        
      
    
    def determine_image_type (self, stream_first_4_bytes):
        """
            Find out the image file type based on the magic number comparison of the first 4 (or 2) bytes
        """
        file_type = None
        bytes_as_hex = b2a_hex(stream_first_4_bytes)
        if bytes_as_hex.startswith('ffd8'):
            file_type = '.jpeg'
        elif bytes_as_hex == '89504e47':
            file_type = '.png'
        elif bytes_as_hex == '47494638':
            file_type = '.gif'
        elif bytes_as_hex.startswith('424d'):
            file_type = '.bmp'
        return file_type
    
    def write_file (self, folder, filename, filedata, flags='w'):
        """
            Write the file data to the folder and filename combination
            (flags: 'w' for write text, 'wb' for write binary, use 'a' instead of 'w' for append)
        """
        result = False
        
        if os.path.isdir(folder):
            try:
                file_obj = open(os.path.join(folder, filename), flags)
                file_obj.write(filedata)
                file_obj.close()
                result = True
            except IOError:
                pass
        return result

    
    def save_image (self, lt_image, page_number, images_folder):
        """
            Try to save the image data from this LTImage object, and return the file name, if successful
        """
        result = None
        if lt_image.stream:
            file_stream = lt_image.stream.get_rawdata()
            
            file_ext = self.determine_image_type(file_stream[0:4])
            
            if file_ext:
                file_name = ''.join([str(page_number), '_', lt_image.name, file_ext])
                if self.write_file(images_folder, file_name, lt_image.stream.get_rawdata(), flags='wb'):
                    result = file_name
            else:
#                file_name = ''.join([str(page_number), '_', lt_image.name, '.tiff'])
#                if self.write_file(images_folder, file_name, lt_image.stream.get_rawdata(), flags='wb'):
#                    result = file_name
                pass
        return result

    def update_page_text_hash (self, h, lt_obj, pct=0.2):
        """
            Use the bbox x0,x1 values within pct% to produce lists of associated text within the hash
        """
        x0 = lt_obj.bbox[0]
        x1 = lt_obj.bbox[2]
        key_found = False
        for k, v in h.items():
            hash_x0 = k[0]
            if x0 >= (hash_x0 * (1.0-pct)) and (hash_x0 * (1.0+pct)) >= x0:
                hash_x1 = k[1]
                if x1 >= (hash_x1 * (1.0-pct)) and (hash_x1 * (1.0+pct)) >= x1:
                    # the text inside this LT* object was positioned at the same
                    # width as a prior series of text, so it belongs together
                    key_found = True
                    v.append(lt_obj.text)
                    h[k] = v
        if not key_found:
            # the text, based on width, is a new series,
            # so it gets its own series (entry in the hash)
           
            h[(x0,x1)] = [lt_obj.text]
        return h

        
    def parse_lt_objs (self, lt_objs, page_number, images_folder, text=[], is_save_image=False):
        """
        Iterate through the list of LT* objects and capture the text or image data contained in each
        """
        text_content = [] 
        page_text = {} # k=(x0, x1) of the bbox, v=list of text strings within  
        for lt_obj in lt_objs:
            
            if isinstance(lt_obj, LTTextBox) or isinstance(lt_obj, LTTextLine):
                # text, so arrange is logically based on its column width
                page_text = self.update_page_text_hash(page_text, lt_obj)

            elif isinstance(lt_obj, LTImage) and is_save_image:
                # an image, so save it to the designated folder, and note it's place in the text
                saved_file = self.save_image(lt_obj, page_number, images_folder)
                if saved_file:
                    # use html style <img /> tag to mark the position of the image within the text
                    text_content.append('<img src="'+os.path.join(images_folder, saved_file)+'" />')
                else:
                    print >> sys.stderr, "Error saving image on page", page_number, lt_obj.__repr__
            elif isinstance(lt_obj, LTFigure):
                # LTFigure objects are containers for other LT* objects, so recurse through the children
                text_content.append(self.parse_lt_objs(lt_obj._objs, page_number, images_folder, text_content))
        
        for k, v in sorted([(key,value) for (key,value) in page_text.items()]):
            # sort the page_text hash by the keys (x0,x1 values of the bbox),
            # which produces a top-down, left-to-right sequence of related columns
            text_content.append('\n'.join(v))

        return '\n'.join(text_content)
    
    def getPage(self, images_folder='./tmp'):
        # receive the LTPage object for the page.
        text_content = [] # a list of strings, each representing text collected from each page of the doc
        # Set parameters for analysis.
        rsrcmgr = PDFResourceManager()
        laparams = LAParams()
        # Create a PDF page aggregator object.
        device = PDFPageAggregator(rsrcmgr, laparams=laparams)
        interpreter = PDFPageInterpreter(rsrcmgr, device)

        for i, page in enumerate(self.doc.get_pages()):
            interpreter.process_page(page)
            # receive the LTPage object for this page
            layout = device.get_result()
            if layout:
                
                text_content.append(self.parse_lt_objs(layout._objs, (i+1), images_folder))
        return text_content
    
    


    def to_bytestring (s, enc='utf-8'):
        """
            Convert the given unicode string to a bytestring, using the standard encoding,
            unless it's already a bytestring
        """
        if s:
            if isinstance(s, str):
                return s
            else:
                return s.encode(enc)
           
    
    def getToc(self):
        '''
        get the table of content
        '''
        toc = []
        try:
            outlines = self.doc.get_outlines()
            for (level,title,dest,a,se) in outlines:
                toc.append( (level, title) )
        except PDFNoOutlines:
            pass
        return toc


class JGParser:
    '''
     We use the BeautifulSoup anlysis the google translate HTML,
     the result is in the <span id='result_box'><span>XXX</span></span>
    '''
    def __init__(self):
        pass
    
    def feed(self, html_data, text):
        soup = MinimalSoup(html_data)
        try:
            
            result = soup('span', title=text)[0].string
        except IndexError:
            result = ""
            
        #for span in soup('span'): # if we use 'in' for iterator, the order of span is vare
        soup = soup('span')
        for i in range(len(soup)):
            if soup[i].string and soup[i].has_key('title'):
                result = result+soup[i].string     
        return result
        #return soup('span', id="result_box")[0].span.string
            
class JGTranslate:
    # it is confortable for splitting to 20 page to batch translate, 
    # if too more then will lost some content , 
    # if too less then will take long time
    def __init__(self, strs, ifilename, ofilename, length=20): 
        fn = open(ifilename, 'w')
        fn1 = open(ofilename, 'w')
        if isinstance(strs, (list,dict)):
            for i in range(len(strs)/length):
                text = ''.join(strs[i*length:(i+1)*length])
                text = ' '.join(text.split())
                fixed_length_print(text, fn)
                try:
                    tr = self.translate(text.strip(), 'en', 'zh-CN')
                    fixed_length_print(tr, fn1, 50)
                except AttributeError:
                    print "JGTranslate error for %s" % text
            if len(strs)>len(strs)/length*length:
                text = ''.join(strs[len(strs)/length*length:])
                text = ' '.join(text.split())
                fixed_length_print(text, fn)
                try:
                    tr = self.translate(text.strip(), 'en', 'zh-CN')
                    fixed_length_print(tr, fn1, 50)
                except AttributeError:
                    print "JGTranslate error for %s" % text
        else:
            fixed_length_print(strs, fn)
            try:
                tr = self.translate(strs.strip(), 'en', 'zh-CN')
                fixed_length_print(tr, fn1, 50)
            except AttributeError:
                print "JGTranslate error for %s" % strs
        fn.close()
        fn1.close()

       
    def translate(self, text, lin, lout):
        '''
        text: the content of the text
        lin: the orign-language of text
        lout: the dest-language of text
        '''
        values={'hl':'zh-CN','ie':'UTF-8','text':text,'sl':lin,'tl':lout}
        
        url='http://translate.google.com.hk/translate_t'
        data = urllib.urlencode(values)
        req = urllib2.Request(url, data)
        req.add_header('User-Agent', 'Mozilla/4.0(compatible; MSIE 6.0; Windows NT 5.1; SV1; NET CLR 2.0.50727)')
        
        respone = urllib2.urlopen(req)
        return JGParser().feed(respone.read(), text)

class JNLTK:
    def __init__(self):
        pass
    
    def tokenize(self, text):
        return sent_tokenize(text)

def fixed_length_print(str, f, fixed_length=100):  
    for i in range(len(str)/fixed_length):
        print >> f, str[i*fixed_length:(i+1)*fixed_length]
    if len(str)>len(str)/fixed_length*fixed_length:
        print >> f, str[len(str)/fixed_length*fixed_length:]
    print >> f, '>>'*50

if __name__ == '__main__':
    
    x = JGPdf("3.pdf").getPage()
    JGTranslate(x, '3.txt', '3_zh.txt', 20)
  
